{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_kg_hide-input": true,
    "_kg_hide-output": true,
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/usst/anaconda3/envs/dxzpy/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.dtype size changed, may indicate binary incompatibility. Expected 96, got 88\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "import random\n",
    "from os import listdir, makedirs\n",
    "from os.path import join, exists, expanduser\n",
    "from tqdm import tqdm\n",
    "\n",
    "import torch\n",
    "from torchvision import transforms\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "import torchvision.models as models\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_uuid": "83ccac88667431f811de838b7b456bb9e41e2060",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#1. set random.seed\n",
    "import random \n",
    "seed = 34\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)\n",
    "torch.backends.cudnn.benchmark = True\n",
    "use_gpu = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_uuid": "3898ed39849e8b4f84baae29bfd800db25a3c250"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31718\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>disease_class</th>\n",
       "      <th>image_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>62fd8bf4d53a1b94fbac16738406f10b.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0bdec5cccbcade6b6e94087cb5509d98.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>8951e940341f77c8d361c1872c67b16d.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>7ed158da58c451f75fb790530d6f19cc.jpg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>9b7399aa-1c3c-4137-ae4e-196cd23fe573___FREC_Sc...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   disease_class                                           image_id\n",
       "0              1               62fd8bf4d53a1b94fbac16738406f10b.jpg\n",
       "1              1               0bdec5cccbcade6b6e94087cb5509d98.jpg\n",
       "2              1               8951e940341f77c8d361c1872c67b16d.jpg\n",
       "3              1               7ed158da58c451f75fb790530d6f19cc.jpg\n",
       "4              1  9b7399aa-1c3c-4137-ae4e-196cd23fe573___FREC_Sc..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = \"./\"\n",
    "train_path = data_path+'train/train/images/'\n",
    "val_path = data_path+'val/val/images/'\n",
    "\n",
    "train=pd.read_json(data_path+\"/train.json\")\n",
    "print(len(train))\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_kg_hide-input": true,
    "_uuid": "dbc3c6a18e147494d2589d33949f1bb59934aebd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "disease_class:61\n",
      "{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60}\n",
      "[1185, 211, 152, 427, 142, 40, 598, 116, 110, 376, 191, 167, 483, 355, 208, 498, 815, 294, 381, 462, 503, 419, 61, 630, 367, 1828, 1799, 251, 857, 770, 1025, 287, 377, 1430, 203, 510, 251, 446, 242, 192, 583, 1208, 319, 966, 1, 1, 251, 442, 264, 1109, 325, 336, 43, 22, 421, 807, 542, 271, 1414, 2473, 261]\n"
     ]
    }
   ],
   "source": [
    "n_train = len(train)\n",
    "categories = set(train['disease_class'])\n",
    "n_class = len(categories)\n",
    "print('disease_class:{}\\n{}'.format(n_class,categories))\n",
    "\n",
    "number_of_classes = []\n",
    "for i in range(n_class):\n",
    "    number_of_classes.append(list(train['disease_class']).count(i))\n",
    "print(number_of_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_uuid": "e5f6f81500e4cfcd8ade95ec61b93fe574ee87fb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[169, 30, 22, 61, 20, 6, 85, 12, 18, 54, 27, 24, 69, 51, 29, 71, 116, 42, 54, 66, 74, 59, 9, 90, 52, 269, 262, 36, 122, 110, 147, 40, 54, 204, 29, 73, 36, 64, 35, 27, 83, 173, 46, 138, 1, 0, 36, 63, 38, 158, 46, 48, 4, 5, 60, 115, 77, 39, 202, 353, 37]\n"
     ]
    }
   ],
   "source": [
    "val = pd.read_json(data_path+\"/val.json\")\n",
    "n_val = len(val)\n",
    "number_of_classes = []\n",
    "for i in range(n_class):\n",
    "    number_of_classes.append(list(val['disease_class']).count(i))\n",
    "print(number_of_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "_uuid": "5eaf9968b312a8ee5ce50851abf262bb19a0b7cb"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/usst/anaconda3/envs/dxzpy/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/models/densenet.py:212: UserWarning: nn.init.kaiming_normal is now deprecated in favor of nn.init.kaiming_normal_.\n"
     ]
    }
   ],
   "source": [
    "num_class = n_class\n",
    "model = models.densenet201()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "_uuid": "f5331850f655f96f61762ef20fe34b2616d18145",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for para in list(model.parameters()):\n",
    "    para.requires_grad=False\n",
    "for para in list(model.features.denseblock3.parameters()):\n",
    "    para.requires_grad=True\n",
    "for para in list(model.features.transition3.parameters()):\n",
    "    para.requires_grad=True\n",
    "for para in list(model.features.denseblock4.parameters()):\n",
    "    para.requires_grad=True\n",
    "for para in list(model.features.norm5.parameters()):\n",
    "    para.requires_grad=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "_uuid": "92c0b95aa1909737a42f04334aadee1701a85f8a",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.classifier = nn.Sequential(\n",
    "    nn.Dropout(0.5),\n",
    "    nn.Linear(1920, num_class),\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "_uuid": "c0dcbd31e9f66b8dc61233f1322a84bee00357f3",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "device_ids = [0,1]\n",
    "\n",
    "if use_gpu:\n",
    "    model = model.cuda(device_ids[0])\n",
    "    model = nn.DataParallel(model, device_ids=device_ids)\n",
    "model.load_state_dict(torch.load('densenet201 8757.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "_uuid": "f7427e653c25f731f184cecaf613540a1d9b1362",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MyDataset(Dataset):\n",
    "    def __init__(self, df_data, data_dir = './', transform=None):\n",
    "        super().__init__()\n",
    "        self.df = df_data.values\n",
    "        self.data_dir = data_dir\n",
    "        self.transform = transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        label,img_name = self.df[index]\n",
    "        img_path = os.path.join(self.data_dir, img_name)\n",
    "        with Image.open(img_path) as img:\n",
    "            image = img.convert('RGB')\n",
    "        if self.transform is not None:\n",
    "            image = self.transform(image)\n",
    "        return image, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "_uuid": "409530b4b870ce901ca1b6a08a4c797f94a6f70e",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "\n",
    "trans_train = transforms.Compose([transforms.RandomResizedCrop(size=224),\n",
    "                                  transforms.RandomHorizontalFlip(),\n",
    "                                  transforms.RandomRotation(30),\n",
    "                                  transforms.ToTensor(),\n",
    "                                  transforms.Normalize(mean=[0.47954108864506007, 0.5295650244021952, 0.39169756009537665],\n",
    "                                                       std=[0.21481591229053462, 0.20095268035289796, 0.24845895286079178])])\n",
    "\n",
    "trans_valid = transforms.Compose([transforms.Resize(size=224),\n",
    "                                  transforms.CenterCrop(size=224),\n",
    "                                  transforms.ToTensor(),\n",
    "                                  transforms.Normalize(mean=[0.47954108864506007, 0.5295650244021952, 0.39169756009537665],\n",
    "                                                       std=[0.21481591229053462, 0.20095268035289796, 0.24845895286079178])])\n",
    "\n",
    "dataset_train = MyDataset(df_data=train, \n",
    "    data_dir=train_path, transform=trans_train)\n",
    "dataset_valid = MyDataset(df_data=val, \n",
    "    data_dir=val_path, transform=trans_valid)\n",
    "\n",
    "loader_train = DataLoader(dataset = dataset_train, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "loader_valid = DataLoader(dataset = dataset_valid, batch_size=64, shuffle=False, num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "_uuid": "02442b06c6eefedb97de2dcb843ad2ded8b2e4c1",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t module.features.denseblock3.denselayer1.norm1.weight\n",
      "\t module.features.denseblock3.denselayer1.norm1.bias\n",
      "\t module.features.denseblock3.denselayer1.conv1.weight\n",
      "\t module.features.denseblock3.denselayer1.norm2.weight\n",
      "\t module.features.denseblock3.denselayer1.norm2.bias\n",
      "\t module.features.denseblock3.denselayer1.conv2.weight\n",
      "\t module.features.denseblock3.denselayer2.norm1.weight\n",
      "\t module.features.denseblock3.denselayer2.norm1.bias\n",
      "\t module.features.denseblock3.denselayer2.conv1.weight\n",
      "\t module.features.denseblock3.denselayer2.norm2.weight\n",
      "\t module.features.denseblock3.denselayer2.norm2.bias\n",
      "\t module.features.denseblock3.denselayer2.conv2.weight\n",
      "\t module.features.denseblock3.denselayer3.norm1.weight\n",
      "\t module.features.denseblock3.denselayer3.norm1.bias\n",
      "\t module.features.denseblock3.denselayer3.conv1.weight\n",
      "\t module.features.denseblock3.denselayer3.norm2.weight\n",
      "\t module.features.denseblock3.denselayer3.norm2.bias\n",
      "\t module.features.denseblock3.denselayer3.conv2.weight\n",
      "\t module.features.denseblock3.denselayer4.norm1.weight\n",
      "\t module.features.denseblock3.denselayer4.norm1.bias\n",
      "\t module.features.denseblock3.denselayer4.conv1.weight\n",
      "\t module.features.denseblock3.denselayer4.norm2.weight\n",
      "\t module.features.denseblock3.denselayer4.norm2.bias\n",
      "\t module.features.denseblock3.denselayer4.conv2.weight\n",
      "\t module.features.denseblock3.denselayer5.norm1.weight\n",
      "\t module.features.denseblock3.denselayer5.norm1.bias\n",
      "\t module.features.denseblock3.denselayer5.conv1.weight\n",
      "\t module.features.denseblock3.denselayer5.norm2.weight\n",
      "\t module.features.denseblock3.denselayer5.norm2.bias\n",
      "\t module.features.denseblock3.denselayer5.conv2.weight\n",
      "\t module.features.denseblock3.denselayer6.norm1.weight\n",
      "\t module.features.denseblock3.denselayer6.norm1.bias\n",
      "\t module.features.denseblock3.denselayer6.conv1.weight\n",
      "\t module.features.denseblock3.denselayer6.norm2.weight\n",
      "\t module.features.denseblock3.denselayer6.norm2.bias\n",
      "\t module.features.denseblock3.denselayer6.conv2.weight\n",
      "\t module.features.denseblock3.denselayer7.norm1.weight\n",
      "\t module.features.denseblock3.denselayer7.norm1.bias\n",
      "\t module.features.denseblock3.denselayer7.conv1.weight\n",
      "\t module.features.denseblock3.denselayer7.norm2.weight\n",
      "\t module.features.denseblock3.denselayer7.norm2.bias\n",
      "\t module.features.denseblock3.denselayer7.conv2.weight\n",
      "\t module.features.denseblock3.denselayer8.norm1.weight\n",
      "\t module.features.denseblock3.denselayer8.norm1.bias\n",
      "\t module.features.denseblock3.denselayer8.conv1.weight\n",
      "\t module.features.denseblock3.denselayer8.norm2.weight\n",
      "\t module.features.denseblock3.denselayer8.norm2.bias\n",
      "\t module.features.denseblock3.denselayer8.conv2.weight\n",
      "\t module.features.denseblock3.denselayer9.norm1.weight\n",
      "\t module.features.denseblock3.denselayer9.norm1.bias\n",
      "\t module.features.denseblock3.denselayer9.conv1.weight\n",
      "\t module.features.denseblock3.denselayer9.norm2.weight\n",
      "\t module.features.denseblock3.denselayer9.norm2.bias\n",
      "\t module.features.denseblock3.denselayer9.conv2.weight\n",
      "\t module.features.denseblock3.denselayer10.norm1.weight\n",
      "\t module.features.denseblock3.denselayer10.norm1.bias\n",
      "\t module.features.denseblock3.denselayer10.conv1.weight\n",
      "\t module.features.denseblock3.denselayer10.norm2.weight\n",
      "\t module.features.denseblock3.denselayer10.norm2.bias\n",
      "\t module.features.denseblock3.denselayer10.conv2.weight\n",
      "\t module.features.denseblock3.denselayer11.norm1.weight\n",
      "\t module.features.denseblock3.denselayer11.norm1.bias\n",
      "\t module.features.denseblock3.denselayer11.conv1.weight\n",
      "\t module.features.denseblock3.denselayer11.norm2.weight\n",
      "\t module.features.denseblock3.denselayer11.norm2.bias\n",
      "\t module.features.denseblock3.denselayer11.conv2.weight\n",
      "\t module.features.denseblock3.denselayer12.norm1.weight\n",
      "\t module.features.denseblock3.denselayer12.norm1.bias\n",
      "\t module.features.denseblock3.denselayer12.conv1.weight\n",
      "\t module.features.denseblock3.denselayer12.norm2.weight\n",
      "\t module.features.denseblock3.denselayer12.norm2.bias\n",
      "\t module.features.denseblock3.denselayer12.conv2.weight\n",
      "\t module.features.denseblock3.denselayer13.norm1.weight\n",
      "\t module.features.denseblock3.denselayer13.norm1.bias\n",
      "\t module.features.denseblock3.denselayer13.conv1.weight\n",
      "\t module.features.denseblock3.denselayer13.norm2.weight\n",
      "\t module.features.denseblock3.denselayer13.norm2.bias\n",
      "\t module.features.denseblock3.denselayer13.conv2.weight\n",
      "\t module.features.denseblock3.denselayer14.norm1.weight\n",
      "\t module.features.denseblock3.denselayer14.norm1.bias\n",
      "\t module.features.denseblock3.denselayer14.conv1.weight\n",
      "\t module.features.denseblock3.denselayer14.norm2.weight\n",
      "\t module.features.denseblock3.denselayer14.norm2.bias\n",
      "\t module.features.denseblock3.denselayer14.conv2.weight\n",
      "\t module.features.denseblock3.denselayer15.norm1.weight\n",
      "\t module.features.denseblock3.denselayer15.norm1.bias\n",
      "\t module.features.denseblock3.denselayer15.conv1.weight\n",
      "\t module.features.denseblock3.denselayer15.norm2.weight\n",
      "\t module.features.denseblock3.denselayer15.norm2.bias\n",
      "\t module.features.denseblock3.denselayer15.conv2.weight\n",
      "\t module.features.denseblock3.denselayer16.norm1.weight\n",
      "\t module.features.denseblock3.denselayer16.norm1.bias\n",
      "\t module.features.denseblock3.denselayer16.conv1.weight\n",
      "\t module.features.denseblock3.denselayer16.norm2.weight\n",
      "\t module.features.denseblock3.denselayer16.norm2.bias\n",
      "\t module.features.denseblock3.denselayer16.conv2.weight\n",
      "\t module.features.denseblock3.denselayer17.norm1.weight\n",
      "\t module.features.denseblock3.denselayer17.norm1.bias\n",
      "\t module.features.denseblock3.denselayer17.conv1.weight\n",
      "\t module.features.denseblock3.denselayer17.norm2.weight\n",
      "\t module.features.denseblock3.denselayer17.norm2.bias\n",
      "\t module.features.denseblock3.denselayer17.conv2.weight\n",
      "\t module.features.denseblock3.denselayer18.norm1.weight\n",
      "\t module.features.denseblock3.denselayer18.norm1.bias\n",
      "\t module.features.denseblock3.denselayer18.conv1.weight\n",
      "\t module.features.denseblock3.denselayer18.norm2.weight\n",
      "\t module.features.denseblock3.denselayer18.norm2.bias\n",
      "\t module.features.denseblock3.denselayer18.conv2.weight\n",
      "\t module.features.denseblock3.denselayer19.norm1.weight\n",
      "\t module.features.denseblock3.denselayer19.norm1.bias\n",
      "\t module.features.denseblock3.denselayer19.conv1.weight\n",
      "\t module.features.denseblock3.denselayer19.norm2.weight\n",
      "\t module.features.denseblock3.denselayer19.norm2.bias\n",
      "\t module.features.denseblock3.denselayer19.conv2.weight\n",
      "\t module.features.denseblock3.denselayer20.norm1.weight\n",
      "\t module.features.denseblock3.denselayer20.norm1.bias\n",
      "\t module.features.denseblock3.denselayer20.conv1.weight\n",
      "\t module.features.denseblock3.denselayer20.norm2.weight\n",
      "\t module.features.denseblock3.denselayer20.norm2.bias\n",
      "\t module.features.denseblock3.denselayer20.conv2.weight\n",
      "\t module.features.denseblock3.denselayer21.norm1.weight\n",
      "\t module.features.denseblock3.denselayer21.norm1.bias\n",
      "\t module.features.denseblock3.denselayer21.conv1.weight\n",
      "\t module.features.denseblock3.denselayer21.norm2.weight\n",
      "\t module.features.denseblock3.denselayer21.norm2.bias\n",
      "\t module.features.denseblock3.denselayer21.conv2.weight\n",
      "\t module.features.denseblock3.denselayer22.norm1.weight\n",
      "\t module.features.denseblock3.denselayer22.norm1.bias\n",
      "\t module.features.denseblock3.denselayer22.conv1.weight\n",
      "\t module.features.denseblock3.denselayer22.norm2.weight\n",
      "\t module.features.denseblock3.denselayer22.norm2.bias\n",
      "\t module.features.denseblock3.denselayer22.conv2.weight\n",
      "\t module.features.denseblock3.denselayer23.norm1.weight\n",
      "\t module.features.denseblock3.denselayer23.norm1.bias\n",
      "\t module.features.denseblock3.denselayer23.conv1.weight\n",
      "\t module.features.denseblock3.denselayer23.norm2.weight\n",
      "\t module.features.denseblock3.denselayer23.norm2.bias\n",
      "\t module.features.denseblock3.denselayer23.conv2.weight\n",
      "\t module.features.denseblock3.denselayer24.norm1.weight\n",
      "\t module.features.denseblock3.denselayer24.norm1.bias\n",
      "\t module.features.denseblock3.denselayer24.conv1.weight\n",
      "\t module.features.denseblock3.denselayer24.norm2.weight\n",
      "\t module.features.denseblock3.denselayer24.norm2.bias\n",
      "\t module.features.denseblock3.denselayer24.conv2.weight\n",
      "\t module.features.denseblock3.denselayer25.norm1.weight\n",
      "\t module.features.denseblock3.denselayer25.norm1.bias\n",
      "\t module.features.denseblock3.denselayer25.conv1.weight\n",
      "\t module.features.denseblock3.denselayer25.norm2.weight\n",
      "\t module.features.denseblock3.denselayer25.norm2.bias\n",
      "\t module.features.denseblock3.denselayer25.conv2.weight\n",
      "\t module.features.denseblock3.denselayer26.norm1.weight\n",
      "\t module.features.denseblock3.denselayer26.norm1.bias\n",
      "\t module.features.denseblock3.denselayer26.conv1.weight\n",
      "\t module.features.denseblock3.denselayer26.norm2.weight\n",
      "\t module.features.denseblock3.denselayer26.norm2.bias\n",
      "\t module.features.denseblock3.denselayer26.conv2.weight\n",
      "\t module.features.denseblock3.denselayer27.norm1.weight\n",
      "\t module.features.denseblock3.denselayer27.norm1.bias\n",
      "\t module.features.denseblock3.denselayer27.conv1.weight\n",
      "\t module.features.denseblock3.denselayer27.norm2.weight\n",
      "\t module.features.denseblock3.denselayer27.norm2.bias\n",
      "\t module.features.denseblock3.denselayer27.conv2.weight\n",
      "\t module.features.denseblock3.denselayer28.norm1.weight\n",
      "\t module.features.denseblock3.denselayer28.norm1.bias\n",
      "\t module.features.denseblock3.denselayer28.conv1.weight\n",
      "\t module.features.denseblock3.denselayer28.norm2.weight\n",
      "\t module.features.denseblock3.denselayer28.norm2.bias\n",
      "\t module.features.denseblock3.denselayer28.conv2.weight\n",
      "\t module.features.denseblock3.denselayer29.norm1.weight\n",
      "\t module.features.denseblock3.denselayer29.norm1.bias\n",
      "\t module.features.denseblock3.denselayer29.conv1.weight\n",
      "\t module.features.denseblock3.denselayer29.norm2.weight\n",
      "\t module.features.denseblock3.denselayer29.norm2.bias\n",
      "\t module.features.denseblock3.denselayer29.conv2.weight\n",
      "\t module.features.denseblock3.denselayer30.norm1.weight\n",
      "\t module.features.denseblock3.denselayer30.norm1.bias\n",
      "\t module.features.denseblock3.denselayer30.conv1.weight\n",
      "\t module.features.denseblock3.denselayer30.norm2.weight\n",
      "\t module.features.denseblock3.denselayer30.norm2.bias\n",
      "\t module.features.denseblock3.denselayer30.conv2.weight\n",
      "\t module.features.denseblock3.denselayer31.norm1.weight\n",
      "\t module.features.denseblock3.denselayer31.norm1.bias\n",
      "\t module.features.denseblock3.denselayer31.conv1.weight\n",
      "\t module.features.denseblock3.denselayer31.norm2.weight\n",
      "\t module.features.denseblock3.denselayer31.norm2.bias\n",
      "\t module.features.denseblock3.denselayer31.conv2.weight\n",
      "\t module.features.denseblock3.denselayer32.norm1.weight\n",
      "\t module.features.denseblock3.denselayer32.norm1.bias\n",
      "\t module.features.denseblock3.denselayer32.conv1.weight\n",
      "\t module.features.denseblock3.denselayer32.norm2.weight\n",
      "\t module.features.denseblock3.denselayer32.norm2.bias\n",
      "\t module.features.denseblock3.denselayer32.conv2.weight\n",
      "\t module.features.denseblock3.denselayer33.norm1.weight\n",
      "\t module.features.denseblock3.denselayer33.norm1.bias\n",
      "\t module.features.denseblock3.denselayer33.conv1.weight\n",
      "\t module.features.denseblock3.denselayer33.norm2.weight\n",
      "\t module.features.denseblock3.denselayer33.norm2.bias\n",
      "\t module.features.denseblock3.denselayer33.conv2.weight\n",
      "\t module.features.denseblock3.denselayer34.norm1.weight\n",
      "\t module.features.denseblock3.denselayer34.norm1.bias\n",
      "\t module.features.denseblock3.denselayer34.conv1.weight\n",
      "\t module.features.denseblock3.denselayer34.norm2.weight\n",
      "\t module.features.denseblock3.denselayer34.norm2.bias\n",
      "\t module.features.denseblock3.denselayer34.conv2.weight\n",
      "\t module.features.denseblock3.denselayer35.norm1.weight\n",
      "\t module.features.denseblock3.denselayer35.norm1.bias\n",
      "\t module.features.denseblock3.denselayer35.conv1.weight\n",
      "\t module.features.denseblock3.denselayer35.norm2.weight\n",
      "\t module.features.denseblock3.denselayer35.norm2.bias\n",
      "\t module.features.denseblock3.denselayer35.conv2.weight\n",
      "\t module.features.denseblock3.denselayer36.norm1.weight\n",
      "\t module.features.denseblock3.denselayer36.norm1.bias\n",
      "\t module.features.denseblock3.denselayer36.conv1.weight\n",
      "\t module.features.denseblock3.denselayer36.norm2.weight\n",
      "\t module.features.denseblock3.denselayer36.norm2.bias\n",
      "\t module.features.denseblock3.denselayer36.conv2.weight\n",
      "\t module.features.denseblock3.denselayer37.norm1.weight\n",
      "\t module.features.denseblock3.denselayer37.norm1.bias\n",
      "\t module.features.denseblock3.denselayer37.conv1.weight\n",
      "\t module.features.denseblock3.denselayer37.norm2.weight\n",
      "\t module.features.denseblock3.denselayer37.norm2.bias\n",
      "\t module.features.denseblock3.denselayer37.conv2.weight\n",
      "\t module.features.denseblock3.denselayer38.norm1.weight\n",
      "\t module.features.denseblock3.denselayer38.norm1.bias\n",
      "\t module.features.denseblock3.denselayer38.conv1.weight\n",
      "\t module.features.denseblock3.denselayer38.norm2.weight\n",
      "\t module.features.denseblock3.denselayer38.norm2.bias\n",
      "\t module.features.denseblock3.denselayer38.conv2.weight\n",
      "\t module.features.denseblock3.denselayer39.norm1.weight\n",
      "\t module.features.denseblock3.denselayer39.norm1.bias\n",
      "\t module.features.denseblock3.denselayer39.conv1.weight\n",
      "\t module.features.denseblock3.denselayer39.norm2.weight\n",
      "\t module.features.denseblock3.denselayer39.norm2.bias\n",
      "\t module.features.denseblock3.denselayer39.conv2.weight\n",
      "\t module.features.denseblock3.denselayer40.norm1.weight\n",
      "\t module.features.denseblock3.denselayer40.norm1.bias\n",
      "\t module.features.denseblock3.denselayer40.conv1.weight\n",
      "\t module.features.denseblock3.denselayer40.norm2.weight\n",
      "\t module.features.denseblock3.denselayer40.norm2.bias\n",
      "\t module.features.denseblock3.denselayer40.conv2.weight\n",
      "\t module.features.denseblock3.denselayer41.norm1.weight\n",
      "\t module.features.denseblock3.denselayer41.norm1.bias\n",
      "\t module.features.denseblock3.denselayer41.conv1.weight\n",
      "\t module.features.denseblock3.denselayer41.norm2.weight\n",
      "\t module.features.denseblock3.denselayer41.norm2.bias\n",
      "\t module.features.denseblock3.denselayer41.conv2.weight\n",
      "\t module.features.denseblock3.denselayer42.norm1.weight\n",
      "\t module.features.denseblock3.denselayer42.norm1.bias\n",
      "\t module.features.denseblock3.denselayer42.conv1.weight\n",
      "\t module.features.denseblock3.denselayer42.norm2.weight\n",
      "\t module.features.denseblock3.denselayer42.norm2.bias\n",
      "\t module.features.denseblock3.denselayer42.conv2.weight\n",
      "\t module.features.denseblock3.denselayer43.norm1.weight\n",
      "\t module.features.denseblock3.denselayer43.norm1.bias\n",
      "\t module.features.denseblock3.denselayer43.conv1.weight\n",
      "\t module.features.denseblock3.denselayer43.norm2.weight\n",
      "\t module.features.denseblock3.denselayer43.norm2.bias\n",
      "\t module.features.denseblock3.denselayer43.conv2.weight\n",
      "\t module.features.denseblock3.denselayer44.norm1.weight\n",
      "\t module.features.denseblock3.denselayer44.norm1.bias\n",
      "\t module.features.denseblock3.denselayer44.conv1.weight\n",
      "\t module.features.denseblock3.denselayer44.norm2.weight\n",
      "\t module.features.denseblock3.denselayer44.norm2.bias\n",
      "\t module.features.denseblock3.denselayer44.conv2.weight\n",
      "\t module.features.denseblock3.denselayer45.norm1.weight\n",
      "\t module.features.denseblock3.denselayer45.norm1.bias\n",
      "\t module.features.denseblock3.denselayer45.conv1.weight\n",
      "\t module.features.denseblock3.denselayer45.norm2.weight\n",
      "\t module.features.denseblock3.denselayer45.norm2.bias\n",
      "\t module.features.denseblock3.denselayer45.conv2.weight\n",
      "\t module.features.denseblock3.denselayer46.norm1.weight\n",
      "\t module.features.denseblock3.denselayer46.norm1.bias\n",
      "\t module.features.denseblock3.denselayer46.conv1.weight\n",
      "\t module.features.denseblock3.denselayer46.norm2.weight\n",
      "\t module.features.denseblock3.denselayer46.norm2.bias\n",
      "\t module.features.denseblock3.denselayer46.conv2.weight\n",
      "\t module.features.denseblock3.denselayer47.norm1.weight\n",
      "\t module.features.denseblock3.denselayer47.norm1.bias\n",
      "\t module.features.denseblock3.denselayer47.conv1.weight\n",
      "\t module.features.denseblock3.denselayer47.norm2.weight\n",
      "\t module.features.denseblock3.denselayer47.norm2.bias\n",
      "\t module.features.denseblock3.denselayer47.conv2.weight\n",
      "\t module.features.denseblock3.denselayer48.norm1.weight\n",
      "\t module.features.denseblock3.denselayer48.norm1.bias\n",
      "\t module.features.denseblock3.denselayer48.conv1.weight\n",
      "\t module.features.denseblock3.denselayer48.norm2.weight\n",
      "\t module.features.denseblock3.denselayer48.norm2.bias\n",
      "\t module.features.denseblock3.denselayer48.conv2.weight\n",
      "\t module.features.transition3.norm.weight\n",
      "\t module.features.transition3.norm.bias\n",
      "\t module.features.transition3.conv.weight\n",
      "\t module.features.denseblock4.denselayer1.norm1.weight\n",
      "\t module.features.denseblock4.denselayer1.norm1.bias\n",
      "\t module.features.denseblock4.denselayer1.conv1.weight\n",
      "\t module.features.denseblock4.denselayer1.norm2.weight\n",
      "\t module.features.denseblock4.denselayer1.norm2.bias\n",
      "\t module.features.denseblock4.denselayer1.conv2.weight\n",
      "\t module.features.denseblock4.denselayer2.norm1.weight\n",
      "\t module.features.denseblock4.denselayer2.norm1.bias\n",
      "\t module.features.denseblock4.denselayer2.conv1.weight\n",
      "\t module.features.denseblock4.denselayer2.norm2.weight\n",
      "\t module.features.denseblock4.denselayer2.norm2.bias\n",
      "\t module.features.denseblock4.denselayer2.conv2.weight\n",
      "\t module.features.denseblock4.denselayer3.norm1.weight\n",
      "\t module.features.denseblock4.denselayer3.norm1.bias\n",
      "\t module.features.denseblock4.denselayer3.conv1.weight\n",
      "\t module.features.denseblock4.denselayer3.norm2.weight\n",
      "\t module.features.denseblock4.denselayer3.norm2.bias\n",
      "\t module.features.denseblock4.denselayer3.conv2.weight\n",
      "\t module.features.denseblock4.denselayer4.norm1.weight\n",
      "\t module.features.denseblock4.denselayer4.norm1.bias\n",
      "\t module.features.denseblock4.denselayer4.conv1.weight\n",
      "\t module.features.denseblock4.denselayer4.norm2.weight\n",
      "\t module.features.denseblock4.denselayer4.norm2.bias\n",
      "\t module.features.denseblock4.denselayer4.conv2.weight\n",
      "\t module.features.denseblock4.denselayer5.norm1.weight\n",
      "\t module.features.denseblock4.denselayer5.norm1.bias\n",
      "\t module.features.denseblock4.denselayer5.conv1.weight\n",
      "\t module.features.denseblock4.denselayer5.norm2.weight\n",
      "\t module.features.denseblock4.denselayer5.norm2.bias\n",
      "\t module.features.denseblock4.denselayer5.conv2.weight\n",
      "\t module.features.denseblock4.denselayer6.norm1.weight\n",
      "\t module.features.denseblock4.denselayer6.norm1.bias\n",
      "\t module.features.denseblock4.denselayer6.conv1.weight\n",
      "\t module.features.denseblock4.denselayer6.norm2.weight\n",
      "\t module.features.denseblock4.denselayer6.norm2.bias\n",
      "\t module.features.denseblock4.denselayer6.conv2.weight\n",
      "\t module.features.denseblock4.denselayer7.norm1.weight\n",
      "\t module.features.denseblock4.denselayer7.norm1.bias\n",
      "\t module.features.denseblock4.denselayer7.conv1.weight\n",
      "\t module.features.denseblock4.denselayer7.norm2.weight\n",
      "\t module.features.denseblock4.denselayer7.norm2.bias\n",
      "\t module.features.denseblock4.denselayer7.conv2.weight\n",
      "\t module.features.denseblock4.denselayer8.norm1.weight\n",
      "\t module.features.denseblock4.denselayer8.norm1.bias\n",
      "\t module.features.denseblock4.denselayer8.conv1.weight\n",
      "\t module.features.denseblock4.denselayer8.norm2.weight\n",
      "\t module.features.denseblock4.denselayer8.norm2.bias\n",
      "\t module.features.denseblock4.denselayer8.conv2.weight\n",
      "\t module.features.denseblock4.denselayer9.norm1.weight\n",
      "\t module.features.denseblock4.denselayer9.norm1.bias\n",
      "\t module.features.denseblock4.denselayer9.conv1.weight\n",
      "\t module.features.denseblock4.denselayer9.norm2.weight\n",
      "\t module.features.denseblock4.denselayer9.norm2.bias\n",
      "\t module.features.denseblock4.denselayer9.conv2.weight\n",
      "\t module.features.denseblock4.denselayer10.norm1.weight\n",
      "\t module.features.denseblock4.denselayer10.norm1.bias\n",
      "\t module.features.denseblock4.denselayer10.conv1.weight\n",
      "\t module.features.denseblock4.denselayer10.norm2.weight\n",
      "\t module.features.denseblock4.denselayer10.norm2.bias\n",
      "\t module.features.denseblock4.denselayer10.conv2.weight\n",
      "\t module.features.denseblock4.denselayer11.norm1.weight\n",
      "\t module.features.denseblock4.denselayer11.norm1.bias\n",
      "\t module.features.denseblock4.denselayer11.conv1.weight\n",
      "\t module.features.denseblock4.denselayer11.norm2.weight\n",
      "\t module.features.denseblock4.denselayer11.norm2.bias\n",
      "\t module.features.denseblock4.denselayer11.conv2.weight\n",
      "\t module.features.denseblock4.denselayer12.norm1.weight\n",
      "\t module.features.denseblock4.denselayer12.norm1.bias\n",
      "\t module.features.denseblock4.denselayer12.conv1.weight\n",
      "\t module.features.denseblock4.denselayer12.norm2.weight\n",
      "\t module.features.denseblock4.denselayer12.norm2.bias\n",
      "\t module.features.denseblock4.denselayer12.conv2.weight\n",
      "\t module.features.denseblock4.denselayer13.norm1.weight\n",
      "\t module.features.denseblock4.denselayer13.norm1.bias\n",
      "\t module.features.denseblock4.denselayer13.conv1.weight\n",
      "\t module.features.denseblock4.denselayer13.norm2.weight\n",
      "\t module.features.denseblock4.denselayer13.norm2.bias\n",
      "\t module.features.denseblock4.denselayer13.conv2.weight\n",
      "\t module.features.denseblock4.denselayer14.norm1.weight\n",
      "\t module.features.denseblock4.denselayer14.norm1.bias\n",
      "\t module.features.denseblock4.denselayer14.conv1.weight\n",
      "\t module.features.denseblock4.denselayer14.norm2.weight\n",
      "\t module.features.denseblock4.denselayer14.norm2.bias\n",
      "\t module.features.denseblock4.denselayer14.conv2.weight\n",
      "\t module.features.denseblock4.denselayer15.norm1.weight\n",
      "\t module.features.denseblock4.denselayer15.norm1.bias\n",
      "\t module.features.denseblock4.denselayer15.conv1.weight\n",
      "\t module.features.denseblock4.denselayer15.norm2.weight\n",
      "\t module.features.denseblock4.denselayer15.norm2.bias\n",
      "\t module.features.denseblock4.denselayer15.conv2.weight\n",
      "\t module.features.denseblock4.denselayer16.norm1.weight\n",
      "\t module.features.denseblock4.denselayer16.norm1.bias\n",
      "\t module.features.denseblock4.denselayer16.conv1.weight\n",
      "\t module.features.denseblock4.denselayer16.norm2.weight\n",
      "\t module.features.denseblock4.denselayer16.norm2.bias\n",
      "\t module.features.denseblock4.denselayer16.conv2.weight\n",
      "\t module.features.denseblock4.denselayer17.norm1.weight\n",
      "\t module.features.denseblock4.denselayer17.norm1.bias\n",
      "\t module.features.denseblock4.denselayer17.conv1.weight\n",
      "\t module.features.denseblock4.denselayer17.norm2.weight\n",
      "\t module.features.denseblock4.denselayer17.norm2.bias\n",
      "\t module.features.denseblock4.denselayer17.conv2.weight\n",
      "\t module.features.denseblock4.denselayer18.norm1.weight\n",
      "\t module.features.denseblock4.denselayer18.norm1.bias\n",
      "\t module.features.denseblock4.denselayer18.conv1.weight\n",
      "\t module.features.denseblock4.denselayer18.norm2.weight\n",
      "\t module.features.denseblock4.denselayer18.norm2.bias\n",
      "\t module.features.denseblock4.denselayer18.conv2.weight\n",
      "\t module.features.denseblock4.denselayer19.norm1.weight\n",
      "\t module.features.denseblock4.denselayer19.norm1.bias\n",
      "\t module.features.denseblock4.denselayer19.conv1.weight\n",
      "\t module.features.denseblock4.denselayer19.norm2.weight\n",
      "\t module.features.denseblock4.denselayer19.norm2.bias\n",
      "\t module.features.denseblock4.denselayer19.conv2.weight\n",
      "\t module.features.denseblock4.denselayer20.norm1.weight\n",
      "\t module.features.denseblock4.denselayer20.norm1.bias\n",
      "\t module.features.denseblock4.denselayer20.conv1.weight\n",
      "\t module.features.denseblock4.denselayer20.norm2.weight\n",
      "\t module.features.denseblock4.denselayer20.norm2.bias\n",
      "\t module.features.denseblock4.denselayer20.conv2.weight\n",
      "\t module.features.denseblock4.denselayer21.norm1.weight\n",
      "\t module.features.denseblock4.denselayer21.norm1.bias\n",
      "\t module.features.denseblock4.denselayer21.conv1.weight\n",
      "\t module.features.denseblock4.denselayer21.norm2.weight\n",
      "\t module.features.denseblock4.denselayer21.norm2.bias\n",
      "\t module.features.denseblock4.denselayer21.conv2.weight\n",
      "\t module.features.denseblock4.denselayer22.norm1.weight\n",
      "\t module.features.denseblock4.denselayer22.norm1.bias\n",
      "\t module.features.denseblock4.denselayer22.conv1.weight\n",
      "\t module.features.denseblock4.denselayer22.norm2.weight\n",
      "\t module.features.denseblock4.denselayer22.norm2.bias\n",
      "\t module.features.denseblock4.denselayer22.conv2.weight\n",
      "\t module.features.denseblock4.denselayer23.norm1.weight\n",
      "\t module.features.denseblock4.denselayer23.norm1.bias\n",
      "\t module.features.denseblock4.denselayer23.conv1.weight\n",
      "\t module.features.denseblock4.denselayer23.norm2.weight\n",
      "\t module.features.denseblock4.denselayer23.norm2.bias\n",
      "\t module.features.denseblock4.denselayer23.conv2.weight\n",
      "\t module.features.denseblock4.denselayer24.norm1.weight\n",
      "\t module.features.denseblock4.denselayer24.norm1.bias\n",
      "\t module.features.denseblock4.denselayer24.conv1.weight\n",
      "\t module.features.denseblock4.denselayer24.norm2.weight\n",
      "\t module.features.denseblock4.denselayer24.norm2.bias\n",
      "\t module.features.denseblock4.denselayer24.conv2.weight\n",
      "\t module.features.denseblock4.denselayer25.norm1.weight\n",
      "\t module.features.denseblock4.denselayer25.norm1.bias\n",
      "\t module.features.denseblock4.denselayer25.conv1.weight\n",
      "\t module.features.denseblock4.denselayer25.norm2.weight\n",
      "\t module.features.denseblock4.denselayer25.norm2.bias\n",
      "\t module.features.denseblock4.denselayer25.conv2.weight\n",
      "\t module.features.denseblock4.denselayer26.norm1.weight\n",
      "\t module.features.denseblock4.denselayer26.norm1.bias\n",
      "\t module.features.denseblock4.denselayer26.conv1.weight\n",
      "\t module.features.denseblock4.denselayer26.norm2.weight\n",
      "\t module.features.denseblock4.denselayer26.norm2.bias\n",
      "\t module.features.denseblock4.denselayer26.conv2.weight\n",
      "\t module.features.denseblock4.denselayer27.norm1.weight\n",
      "\t module.features.denseblock4.denselayer27.norm1.bias\n",
      "\t module.features.denseblock4.denselayer27.conv1.weight\n",
      "\t module.features.denseblock4.denselayer27.norm2.weight\n",
      "\t module.features.denseblock4.denselayer27.norm2.bias\n",
      "\t module.features.denseblock4.denselayer27.conv2.weight\n",
      "\t module.features.denseblock4.denselayer28.norm1.weight\n",
      "\t module.features.denseblock4.denselayer28.norm1.bias\n",
      "\t module.features.denseblock4.denselayer28.conv1.weight\n",
      "\t module.features.denseblock4.denselayer28.norm2.weight\n",
      "\t module.features.denseblock4.denselayer28.norm2.bias\n",
      "\t module.features.denseblock4.denselayer28.conv2.weight\n",
      "\t module.features.denseblock4.denselayer29.norm1.weight\n",
      "\t module.features.denseblock4.denselayer29.norm1.bias\n",
      "\t module.features.denseblock4.denselayer29.conv1.weight\n",
      "\t module.features.denseblock4.denselayer29.norm2.weight\n",
      "\t module.features.denseblock4.denselayer29.norm2.bias\n",
      "\t module.features.denseblock4.denselayer29.conv2.weight\n",
      "\t module.features.denseblock4.denselayer30.norm1.weight\n",
      "\t module.features.denseblock4.denselayer30.norm1.bias\n",
      "\t module.features.denseblock4.denselayer30.conv1.weight\n",
      "\t module.features.denseblock4.denselayer30.norm2.weight\n",
      "\t module.features.denseblock4.denselayer30.norm2.bias\n",
      "\t module.features.denseblock4.denselayer30.conv2.weight\n",
      "\t module.features.denseblock4.denselayer31.norm1.weight\n",
      "\t module.features.denseblock4.denselayer31.norm1.bias\n",
      "\t module.features.denseblock4.denselayer31.conv1.weight\n",
      "\t module.features.denseblock4.denselayer31.norm2.weight\n",
      "\t module.features.denseblock4.denselayer31.norm2.bias\n",
      "\t module.features.denseblock4.denselayer31.conv2.weight\n",
      "\t module.features.denseblock4.denselayer32.norm1.weight\n",
      "\t module.features.denseblock4.denselayer32.norm1.bias\n",
      "\t module.features.denseblock4.denselayer32.conv1.weight\n",
      "\t module.features.denseblock4.denselayer32.norm2.weight\n",
      "\t module.features.denseblock4.denselayer32.norm2.bias\n",
      "\t module.features.denseblock4.denselayer32.conv2.weight\n",
      "\t module.features.norm5.weight\n",
      "\t module.features.norm5.bias\n",
      "\t module.classifier.1.weight\n",
      "\t module.classifier.1.bias\n"
     ]
    }
   ],
   "source": [
    "params_to_update = []\n",
    "for name,param in model.named_parameters():\n",
    "    if param.requires_grad == True:\n",
    "        params_to_update.append(param)\n",
    "        print(\"\\t\",name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "_uuid": "da187c257bd58099f3675ea1947f5976fa6986aa",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_epochs = 20\n",
    "early_stopping = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "_uuid": "06c91ccc718162df65a8976856c203bf971ce7ea",
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cross_entropy(input, target, size_average=True):\n",
    "    \"\"\" Cross entropy that accepts soft targets\n",
    "    Args:\n",
    "         pred: predictions for neural network\n",
    "         targets: targets, can be soft\n",
    "         size_average: if false, sum is returned instead of mean\n",
    "\n",
    "    Examples::\n",
    "\n",
    "        input = torch.FloatTensor([[1.1, 2.8, 1.3], [1.1, 2.1, 4.8]])\n",
    "        input = torch.autograd.Variable(out, requires_grad=True)\n",
    "\n",
    "        target = torch.FloatTensor([[0.05, 0.9, 0.05], [0.05, 0.05, 0.9]])\n",
    "        target = torch.autograd.Variable(y1)\n",
    "        loss = cross_entropy(input, target)\n",
    "        loss.backward()\n",
    "    \"\"\"\n",
    "    logsoftmax = nn.LogSoftmax()\n",
    "    if size_average:\n",
    "        return torch.mean(torch.sum(-target * logsoftmax(input), dim=1))\n",
    "    else:\n",
    "        return torch.sum(torch.sum(-target * logsoftmax(input), dim=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "_uuid": "0ee0b2ab48083f2958779bf7a9738defe8a9fed1",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "."
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/usst/anaconda3/envs/dxzpy/lib/python3.6/site-packages/torch/tensor.py:263: UserWarning: non-inplace resize is deprecated\n",
      "  warnings.warn(\"non-inplace resize is deprecated\")\n",
      "/home/usst/anaconda3/envs/dxzpy/lib/python3.6/site-packages/ipykernel_launcher.py:20: UserWarning: Implicit dimension choice for log_softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".......................................................................................................................................................................................................................................................\n",
      "[Epoch 0] train loss 1.057316 train acc 0.867268  valid loss 1.020904 valid acc 0.874890  time 385.702569\n",
      "........................................................................................................................................................................................................................................................\n",
      "[Epoch 1] train loss 1.058728 train acc 0.865849  valid loss 1.021330 valid acc 0.874449  time 382.449714\n",
      "........................................................................................................................................................................................................................................................\n",
      "[Epoch 2] train loss 1.061733 train acc 0.864903  valid loss 1.020121 valid acc 0.874890  time 379.129229\n",
      "........................................................................................................................................................................................................................................................\n",
      "[Epoch 3] train loss 1.061305 train acc 0.865660  valid loss 1.019529 valid acc 0.875771  time 380.264335\n",
      "save model...\n",
      "saved.\n",
      "........................................................................................................................................................................................................................................................\n",
      "[Epoch 4] train loss 1.056393 train acc 0.865944  valid loss 1.022011 valid acc 0.875110  time 380.824104\n",
      "........................................................................................................................................................................................................................................................\n",
      "[Epoch 5] train loss 1.054995 train acc 0.867615  valid loss 1.020799 valid acc 0.874229  time 381.964436\n",
      "........................................................................................................................................................................................................................................................\n",
      "[Epoch 6] train loss 1.056885 train acc 0.866227  valid loss 1.021436 valid acc 0.875110  time 381.722566\n",
      "........................................................................................................................................................................................................................................................\n",
      "[Epoch 7] train loss 1.058135 train acc 0.865218  valid loss 1.020178 valid acc 0.874229  time 390.246864\n",
      "........................................................................................................................................................................................................................................................\n",
      "[Epoch 8] train loss 1.056503 train acc 0.867425  valid loss 1.020390 valid acc 0.874009  time 382.879676\n",
      "........................................................................................................................................................................................................................................................\n",
      "[Epoch 9] train loss 1.058139 train acc 0.865218  valid loss 1.020801 valid acc 0.873568  time 389.187939\n",
      "Finished Training\n",
      "best_epoch: 3, best_val_acc 0.875771\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "criterion = cross_entropy\n",
    "optimizer = optim.SGD(params_to_update,lr = 1e-5)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.5)\n",
    "label_smoothing = 0.1\n",
    "\n",
    "best_val_acc = 0.875\n",
    "best_epoch = 0\n",
    "epoch_since_best = 0\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    at = time.time()\n",
    "    scheduler.step()\n",
    "    model.train()\n",
    "    train_total_samples = 0\n",
    "    train_acc = 0\n",
    "    train_loss = 0\n",
    "    for i, data in enumerate(loader_train):\n",
    "        print('.',end='')        \n",
    "        inputs, label = data\n",
    "        train_total_samples += label.size()[0]        \n",
    "        labels = label.resize(label.size()[0], 1)\n",
    "        labels = torch.FloatTensor(label.size()[0], n_class).zero_().scatter_(1, labels.resize(label.size()[0], 1) ,1)\n",
    "        labels = (1 - label_smoothing) * labels + (label_smoothing / n_class)\n",
    "        if use_gpu:\n",
    "            inputs, labels, label = inputs.cuda(), labels.cuda(), label.cuda()\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(inputs)        \n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        train_pred = torch.argmax(outputs.data, dim=1)        \n",
    "        train_acc += torch.sum(train_pred == label.data)\n",
    "        train_loss += loss.item() * labels.size()[0]\n",
    "            \n",
    "    model.eval()\n",
    "    valid_total_samples = 0\n",
    "    valid_acc = 0\n",
    "    val_loss = 0\n",
    "    for _, data in enumerate(loader_valid):     \n",
    "        inputs, label = data\n",
    "        valid_total_samples += label.size()[0]        \n",
    "        labels = label\n",
    "        labels = torch.FloatTensor(label.size()[0], n_class).zero_().scatter_(1, labels.resize(label.size()[0], 1) ,1)\n",
    "        labels = (1 - label_smoothing) * labels + (label_smoothing / n_class)\n",
    "        if use_gpu:\n",
    "            inputs, labels, label = inputs.cuda(), labels.cuda(), label.cuda()\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        valid_pred = torch.argmax(outputs.data, dim=1)        \n",
    "        valid_acc += torch.sum(valid_pred == label.data)\n",
    "        val_loss += loss.item() * labels.size()[0]\n",
    "\n",
    "    train_acc = train_acc.cpu().numpy() / train_total_samples\n",
    "    valid_acc = valid_acc.cpu().numpy() / valid_total_samples\n",
    "    train_loss = train_loss / train_total_samples\n",
    "    val_loss = val_loss / valid_total_samples\n",
    "    \n",
    "    print()\n",
    "    bt = time.time()\n",
    "    print('[Epoch %d] train loss %.6f train acc %.6f  valid loss %.6f valid acc %.6f  time %.6f' % (\n",
    "        epoch, train_loss, train_acc, val_loss, valid_acc,bt-at))\n",
    "\n",
    "    if valid_acc > best_val_acc:\n",
    "        best_val_acc = valid_acc\n",
    "        best_epoch = epoch\n",
    "        epoch_since_best = 0\n",
    "        print('save model...')\n",
    "        torch.save(model.state_dict(), 'tuned-densenet201.pth')\n",
    "        print('saved.')\n",
    "    else:\n",
    "        epoch_since_best += 1\n",
    "        \n",
    "    if epoch_since_best > early_stopping:\n",
    "        break\n",
    "            \n",
    "print('Finished Training')\n",
    "print('best_epoch: %d, best_val_acc %.6f' % (best_epoch, best_val_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
